package tech.abing.spider.task.crawler.simuwang;

import java.util.Arrays;
import java.util.List;
import java.util.Map;

import tech.abing.spider.CrawTaskBuilder;
import tech.abing.spider.CrawlManager;
import tech.abing.spider.CrawlTask;
import tech.abing.spider.component.DynamicEntrance;
import tech.abing.spider.component.listener.TaskLifeListener;
import tech.abing.spider.downloader.impl.DefaultPageDownloader;
import tech.abing.spider.request.PageRequest;
import tech.abing.spider.request.PageRequest.Method;
import tech.abing.spider.request.PageRequest.PageEncoding;
import tech.abing.spider.request.StartContext;
import tech.abing.spider.task.constants.DictEnum;
import tech.abing.spider.task.pageprocessor.sumuwang.FundDataProcessor;
import tech.abing.spider.task.pipeline.sumuwang.FundDataPipeline;
import tech.abing.spider.task.proxy.MyProxyIpPool;
import tech.abing.spider.util.SqlUtil;

public class FundDataSpider {

	private static SqlUtil sqlUtil = null;
	
	public static void main(String[] args) {
		CrawlManager crawlManager = CrawlManager.getInstance();
		
		CrawTaskBuilder builder = crawlManager
		.prepareCrawlTask(DictEnum.TASK_TYPE_FUND_DATA.getName(), DefaultPageDownloader.class)
		.useThread(100)
		.useDynamicEntrance(DynamicEntranceImpl.class)
		.usePageEncoding(PageEncoding.UTF8)
		.usePageRetryCount(3)
		.setPolitenessDelay(0)
		.usePipeline(FundDataPipeline.class)
		.useProxyIpPool(MyProxyIpPool.class, 100, 1000 * 60 * 30, -1)//每次网络请求10个代理ip,ip30分钟后过期,有效的ip不限制使用次数
		.useTaskLifeListener(new TaskLifeListener(){

			@Override
			public void onStart(CrawlTask task) {
				System.out.println("================开始");
				if(sqlUtil==null){
					sqlUtil = SqlUtil.getInstance();
				}
				sqlUtil.deleteForConn("delete from smw_fund_data");
				
				sqlUtil.deleteForConn("delete from smw_fund_nav");
			}

			@Override
			public void onFinished(CrawlTask task) {
				task.getPipeline().destory();
				sqlUtil.closeConn();
				System.out.println("================完成");
			}
			
		});
		
		CrawlTask task = builder.build();
		
		crawlManager.start(task);
	}
	
	public static final class DynamicEntranceImpl extends DynamicEntrance{

		@Override
		public List<StartContext> loadStartContext() {
			StartContext context = new StartContext();
			
			String sql = "select DISTINCT PRODUCT_NAME_ABBR from fivp_product_info";
			
			List<Map<String, Object>> mapList = sqlUtil.searchListForConn(sql);
			for(Map<String, Object> map : mapList){
				String productName = map.get("PRODUCT_NAME_ABBR").toString();
				String url = "http://dc.simuwang.com/Screen/getData.html?page=1&condition=fund_type:1,6,4,3,8,2,7,5;sort_name:ret_incep;sort_asc:desc;keyword:"+productName+";ret:ret_incep;";
				
				PageRequest request = context.createPageRequest(url, FundDataProcessor.class);
				request.setMethod(Method.GET);
				
				request.putHeader("Cookie", "rz_token_6658=ba231ebfcdd24e06708e4e5d08751897.1481882878; guest_id=1481883047; regsms=1481882878000; stat_sessid=1sm68o8o72ecvf008e854ipa26; http_tK_cache=c4f592f3899e60e625524a814b9f28bef1b3b80f; cur_ck_time=1481882926; ck_request_key=9LJCaRiczPSCSTehopRPCelGFPZ4TtcrcneeTV9SmNs%3D; passport=281090%09u_476668644256%09VlVQClFTB18BUVAFVVAFVw1WAQcDVQJUBwYBAgUFBVY%3D1b38ea2618; rz_u_p=d41d8cd98f00b204e9800998ecf8427e%3Du_476668644256; rz_rem_u_p=ip7zS1Cvi24LS6tsm%2F%2BasobNd9Bwrq0y5DuZAJclD9A%3D%24zxIAAybf1yufdhQt6M1IKEUQhYg0VrGymP%2BaSvbTukM%3D; smppw_tz_auth=1; had_quiz_281090=1481882928000; PHPSESSID=cs8ufr3ivr5ei2kvvsb6endrr3; passportmall=A1pXBAVRaBNmAQNSUAFZVVFXVAYP7378cf9565; autologin_status=0; fyr_ssid_n5776=fyr_n5776_iwrmu7yx; Hm_lvt_c3f6328a1a952e922e996c667234cdae=1481882878; Hm_lpvt_c3f6328a1a952e922e996c667234cdae=1481884064");
				request.putHeader("Content-Type", "application/json; charset=UTF-8");
				request.putHeader(
						"User-Agent",
						"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36");
				request.putHeader("Accept", "application/json, text/javascript, */*; q=0.01");
				request.putHeader("Accept-Language", "zh-CN,zh;q=0.8");
				request.putHeader("Cache-Control", "no-cache");
				
				request.putHeader("Referer", "http://dc.simuwang.com/screen.html");
				
				request.putHeader("X-Requested-With", "XMLHttpRequest");
				request.putHeader("Host", "dc.simuwang.com");
				
				context.injectSeed(request);
			}
			
			return Arrays.asList(context);
		}
		
	}
}
